#!/usr/bin/env python
# encoding: utf-8
"""
    @author:水痕
    @time:2018-03-28 14:11:39
    @email:332904234@qq.com
    @version:1.0
    @fileName:12-抓取数据存入数据库.py
    @describe:
"""
import collections

from bs4 import BeautifulSoup
from Mysql import Mysql
from util.Redis import Redis
from util.Date import fmt_date
import time, json


import requests


URL_STR = 'https://www.vigeotec.com/sh/query?position={}'
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
    'Content-Type': 'multipart/form-data',
    'Host': 'www.vigeotec.com'
}

REDIS_HASH_KEY_PREFIX = 'REDIS_HASH_'

#查询
def query_database(keyword):
    redis = Redis().get_instance()
    _id = fmt_date(time.time())
    redis_hash_key = REDIS_HASH_KEY_PREFIX + keyword + _id
    print(redis_hash_key)
    if redis.exists(redis_hash_key):
        print(redis.get(redis_hash_key))
        rs = redis.get(redis_hash_key)
        print(type(rs))
        str_list = json.loads(rs)
        print(type(str_list))
        return str_list
    else:
        form_html_datashanghai(keyword)
        db = Mysql()
        results = db.getData(keyword)
        rs = tableToJson(results)
        redis.set(redis_hash_key, rs, ex=300)
        return json.loads(rs)

def tableToJson(results):
    req = []
    for row in results:
        d = collections.OrderedDict()
        d['id'] = row[0]
        d['area_name'] = row[1]
        d['latest_diagnosis_time'] = row[2]
        d['street'] = row[3]
        d['unsealing_time'] = row[4]
        req.append(d)
    return json.dumps(req)


# 获取网页的函数
def form_html_datashanghai(keyword):

    dataform = {'position': keyword}
    urlStr = URL_STR.format(keyword)
    print(urlStr)
    response = requests.post(urlStr, data=dataform, headers=headers)
    response.encoding = 'utf-8'
    if(response.status_code==200):
        soup = BeautifulSoup(response.content, "html5lib")
        article_table = soup.table
        article_tbody = article_table.tbody
        for article_tbody_tr in article_tbody.findAll('tr'):
            article_tbody_tr_td = article_tbody_tr.findAll('td')
            if(len(article_tbody_tr_td)==4):
                area_name = article_tbody_tr_td[0].getText()
                latest_diagnosis_time = article_tbody_tr_td[1].getText()
                street = article_tbody_tr_td[2].getText()
                unsealing_time = article_tbody_tr_td[3].getText()
                db = Mysql()
                db.insert_tableshanghai(area_name, latest_diagnosis_time, street, unsealing_time)
